import urllib.request as ur
import lxml.etree as le
import user_agent
import time

keyword = input('请输入关键词：')
pn_start = int(input('起始页:'))
pn_end = int(input('终止页：'))


def getRequest(url):  # 构建对象
    time.sleep(2)
    return ur.Request(    # 构建请求对象
        url=url,
        headers={
            'User-Agent': user_agent.get_user_agent_pc(),
            'cookie': 'TY_SESSION_ID=9b5b5e23-7593-4d1a-a3e1-b2a91dfa5a14; JSESSIONID=CB8F8A0C4F75562277097AA970BA4465; uuid_tt_dd=10_19704076950-1576030272413-306130; dc_session_id=10_1576030272413.199875; c-login-auto=1; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1576030274; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_19704076950-1576030272413-306130; announcement=%257B%2522isLogin%2522%253Afalse%252C%2522announcementUrl%2522%253A%2522https%253A%252F%252Fblogdev.blog.csdn.net%252Farticle%252Fdetails%252F103053996%2522%252C%2522announcementCount%2522%253A0%252C%2522announcementExpire%2522%253A3600000%257D; acw_tc=2760824c15760302803645334e6c80fb98788a0880d188323611c48cd82b85; dc_tos=q2bsqx; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1576030282',
        }
    )

def getProxyOpener():   # 代理IP  构建打开对象

    return ur.build_opener()

for pn in range(pn_start,pn_end+1):    # 从页面开始循环
    # url = 'https://so.csdn.net/so/search/s.do?p=%s&q=%s&t=&viparticle=&domain=&o=&s=&u=&l=&f=' % (pn,keyword)  # 构建一个url地址
    request = getRequest(
        'https://so.csdn.net/so/search/s.do?p=%s&q=%s&t=&viparticle=&domain=&o=&s=&u=&l=&f=' % (pn,keyword)
    )
    try:
        response = getProxyOpener().open(request).read()
        href_s = le.HTML(response).xpath('//span[@class="down fr"]/../span[@class="link"]/a/@href')
    # print(href_s)
        for href in href_s:
            try:
                response_blog = getProxyOpener().open(
                    getRequest(href)
                ).read()
                title1 = le.HTML(response_blog).xpath('//h1[@class="title-article"]/text()')
                print(title1)
                with open('blog/%s.html','wb') as f:
                    f.write(response_blog)
            except Exception as e:
                print(e)
    except:pass
